﻿#region License
// Author: David Kirkland <davidrkirkland@gmail.com>
// Copyright 2007-2008 Illogikal Software 
//
// Licensed under the Apache License, Version 2.0 (the "License"); 
// you may not use this file except in compliance with the License. 
// You may obtain a copy of the License 
// 
//     http://www.apache.org/licenses/LICENSE-2.0 
//
// Unless required by applicable law or agreed to in writing, software 
// distributed under the License is distributed on an "AS IS" BASIS, 
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
// See the License for the specific language governing permissions and 
// limitations under the License.
#endregion

#region Using Directives
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using VitaminK.Extensions;
#endregion

namespace NetSpider.Policies
{
    public class RobotsTxtParser
    {
        private const string COMMENT_STARTS = "#";
        private const string TOKEN_SEPARATOR = ":";

        private const string USER_AGENT_FIELD = "User-agent";
        private const string DISALLOW_FIELD = "Disallow";

        public RobotPolicy ParsePolicy(string domainName, DateTime retrieved, string policyText)
        {
            RobotPolicy policy = new RobotPolicy(domainName, retrieved);
            string[] lines = policyText.SplitLines(StringSplitOptions.RemoveEmptyEntries);

            string userAgent = null;

            foreach (string line in lines)
            {
                string field;
                string value;

                if (TokenAreValid(line, out field, out value))
                {
                    if (field == USER_AGENT_FIELD)
                    {
                        userAgent = value;
                    }
                    else if (field == DISALLOW_FIELD)
                    {
                        policy.AddDisallow(userAgent, value);
                    }
                }
            }

            return policy;
        }

        private bool TokenAreValid(string line, out string field, out string value)
        {
            field = null;
            value = null;

            string cleanLine = CleanAndRemoveComments(line);
            int tokenSeparatorPosition = cleanLine.IndexOf(TOKEN_SEPARATOR);

            if (cleanLine.Length == 0 || tokenSeparatorPosition == -1 || tokenSeparatorPosition == cleanLine.Length - 1)
            {
                return false;
            }

            field = cleanLine.Substring(0, tokenSeparatorPosition).Trim();
            value = cleanLine.Substring(tokenSeparatorPosition + 1, cleanLine.Length - field.Length - 1).Trim();

            if (value.Length == 0)
            {
                return false;
            }

            return true;
        }

        private string CleanAndRemoveComments(string line)
        {
            int commentStartsAt = line.IndexOf(COMMENT_STARTS);

            if (commentStartsAt == -1)
            {
                return line.Trim();
            }

            return line.Substring(0, commentStartsAt).Trim();
        }
    }
}
